// 该脚本为node.js 编写的爬虫脚本,用于将第三方网站的数据爬取到本地并保存到数据库

var Nightmare = require('nightmare');
var nightmare = Nightmare({
    openDevTools: {
        mode: 'detach'
    },
    show: true,
    pollInterval: 1000
});

var mysql = require('mysql');
var connection = mysql.createConnection({
    host: 'localhost',
    user: 'root',
    password: '123456',
    database: 'web_mojianworld_server'
});

// 连接数据库
connection.connect();

// 爬取数据
nightmare
    .goto('http://www.moto188.com/')
    .wait(function () {
        // 等待页面载入
        return document.getElementsByClassName("hnbox").length > 0;
    })
    .wait(function () {
        let data = [];
        let dom = document.querySelectorAll(".hnbox div[class='f_l']");
        for (let i = 0; i < dom.length; i++) {
            let item = [];
            let dom2 = dom[i].getElementsByTagName("a");
            for (let j = 0; j < dom2.length; j++) {
                item.push(dom2[j].innerText);
            }
            data.push(item);
        }
        window.localStorage["data"] = JSON.stringify(data);
        return true;
    }).evaluate(function () {
    return window.localStorage["data"];
})
// .end()
    .then(function (res) {
        let data = JSON.parse(res);
        for (let i = 0; i < data.length; i++) {
            let item = data[i];
            for (let j = 0; j < item.length; j++) {
                let sql = "insert into web_child_item(parentId,title,orderNum) values(?,?,?)";
                let params = [i + 1, item[j], j + 1];
                connection.query(sql, params, function (err, result) {
                    if (err) {
                        console.log('[INSERT ERROR] - ', err.message);
                        return;
                    }
                    console.log("正在处理:" + JSON.stringify(result));
                })
            }
        }
    }).then(function () {
        // 关闭数据库连接
        console.log("数据爬取完毕");
        connection.end();
    }
).catch(function (error) {
    console.log('错误是：' + error);
})

